{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1.制作停用词"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\3技术研究\\1语料.xlsx')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>PN 专利号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>AD 申请详细信息</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>patent5534</td>\n",
       "      <td>US2015071737-A1;  US10424498-B2</td>\n",
       "      <td>Transport apparatus for vacuum transport syste...</td>\n",
       "      <td>US2015071737-A1    US480803    09 Sep 2014;   ...</td>\n",
       "      <td>transport apparatus vacuum transport system ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>patent31677</td>\n",
       "      <td>CN107498781-A;  CN207290719-U;  CN107498781-B</td>\n",
       "      <td>Automatic injection molding machine feeding   ...</td>\n",
       "      <td>CN107498781-A    CN10664563    04 Aug 2017;   ...</td>\n",
       "      <td>automatic injection mold machine feeding syste...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>patent83440</td>\n",
       "      <td>CN108029284-A;  CN207869642-U</td>\n",
       "      <td>Fully-automatic single-row planting robot, has...</td>\n",
       "      <td>CN108029284-A    CN10027388    11 Jan 2018;   ...</td>\n",
       "      <td>fully-automatic single-row planting robot rota...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>patent10376</td>\n",
       "      <td>CN104785960-A;  CN104785960-B</td>\n",
       "      <td>Grid structure type robot welding device has r...</td>\n",
       "      <td>CN104785960-A    CN10150108    31 Mar 2015;   ...</td>\n",
       "      <td>grid structure type robot weld device robot pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>patent12420</td>\n",
       "      <td>WO2017043583-A1;  TW201718203-A;  JP2017539221...</td>\n",
       "      <td>Robot apparatus for use in industrial applicat...</td>\n",
       "      <td>WO2017043583-A1    WOJP076473    08 Sep 2016; ...</td>\n",
       "      <td>robot apparatus industrial application operati...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            编号                                             PN 专利号  \\\n",
       "0   patent5534                    US2015071737-A1;  US10424498-B2   \n",
       "1  patent31677      CN107498781-A;  CN207290719-U;  CN107498781-B   \n",
       "2  patent83440                      CN108029284-A;  CN207869642-U   \n",
       "3  patent10376                      CN104785960-A;  CN104785960-B   \n",
       "4  patent12420  WO2017043583-A1;  TW201718203-A;  JP2017539221...   \n",
       "\n",
       "                                             TI 文献标题  \\\n",
       "0  Transport apparatus for vacuum transport syste...   \n",
       "1  Automatic injection molding machine feeding   ...   \n",
       "2  Fully-automatic single-row planting robot, has...   \n",
       "3  Grid structure type robot welding device has r...   \n",
       "4  Robot apparatus for use in industrial applicat...   \n",
       "\n",
       "                                           AD 申请详细信息  \\\n",
       "0  US2015071737-A1    US480803    09 Sep 2014;   ...   \n",
       "1  CN107498781-A    CN10664563    04 Aug 2017;   ...   \n",
       "2  CN108029284-A    CN10027388    11 Jan 2018;   ...   \n",
       "3  CN104785960-A    CN10150108    31 Mar 2015;   ...   \n",
       "4  WO2017043583-A1    WOJP076473    08 Sep 2016; ...   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  transport apparatus vacuum transport system ar...  \n",
       "1  automatic injection mold machine feeding syste...  \n",
       "2  fully-automatic single-row planting robot rota...  \n",
       "3  grid structure type robot weld device robot pr...  \n",
       "4  robot apparatus industrial application operati...  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['编号', 'PN 专利号', ' TI 文献标题', ' AD 申请详细信息', '预处理文本'], dtype='object')"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "ls = []\n",
    "for i in data['预处理文本'].values:\n",
    "    for ii in i.split():\n",
    "        ls.append(ii)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8954466"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "result = Counter(ls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "k_ls, v_ls = [], []\n",
    "for k,v in result.items():\n",
    "    k_ls.append(k)\n",
    "    v_ls.append(v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({'词':k_ls, '频次':v_ls})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\3技术研究\\词和词频.csv',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>PN 专利号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>AD 申请详细信息</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>patent5534</td>\n",
       "      <td>US2015071737-A1;  US10424498-B2</td>\n",
       "      <td>Transport apparatus for vacuum transport syste...</td>\n",
       "      <td>US2015071737-A1    US480803    09 Sep 2014;   ...</td>\n",
       "      <td>transport apparatus vacuum transport system ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>patent31677</td>\n",
       "      <td>CN107498781-A;  CN207290719-U;  CN107498781-B</td>\n",
       "      <td>Automatic injection molding machine feeding   ...</td>\n",
       "      <td>CN107498781-A    CN10664563    04 Aug 2017;   ...</td>\n",
       "      <td>automatic injection mold machine feeding syste...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>patent83440</td>\n",
       "      <td>CN108029284-A;  CN207869642-U</td>\n",
       "      <td>Fully-automatic single-row planting robot, has...</td>\n",
       "      <td>CN108029284-A    CN10027388    11 Jan 2018;   ...</td>\n",
       "      <td>fully-automatic single-row planting robot rota...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>patent10376</td>\n",
       "      <td>CN104785960-A;  CN104785960-B</td>\n",
       "      <td>Grid structure type robot welding device has r...</td>\n",
       "      <td>CN104785960-A    CN10150108    31 Mar 2015;   ...</td>\n",
       "      <td>grid structure type robot weld device robot pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>patent12420</td>\n",
       "      <td>WO2017043583-A1;  TW201718203-A;  JP2017539221...</td>\n",
       "      <td>Robot apparatus for use in industrial applicat...</td>\n",
       "      <td>WO2017043583-A1    WOJP076473    08 Sep 2016; ...</td>\n",
       "      <td>robot apparatus industrial application operati...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            编号                                             PN 专利号  \\\n",
       "0   patent5534                    US2015071737-A1;  US10424498-B2   \n",
       "1  patent31677      CN107498781-A;  CN207290719-U;  CN107498781-B   \n",
       "2  patent83440                      CN108029284-A;  CN207869642-U   \n",
       "3  patent10376                      CN104785960-A;  CN104785960-B   \n",
       "4  patent12420  WO2017043583-A1;  TW201718203-A;  JP2017539221...   \n",
       "\n",
       "                                             TI 文献标题  \\\n",
       "0  Transport apparatus for vacuum transport syste...   \n",
       "1  Automatic injection molding machine feeding   ...   \n",
       "2  Fully-automatic single-row planting robot, has...   \n",
       "3  Grid structure type robot welding device has r...   \n",
       "4  Robot apparatus for use in industrial applicat...   \n",
       "\n",
       "                                           AD 申请详细信息  \\\n",
       "0  US2015071737-A1    US480803    09 Sep 2014;   ...   \n",
       "1  CN107498781-A    CN10664563    04 Aug 2017;   ...   \n",
       "2  CN108029284-A    CN10027388    11 Jan 2018;   ...   \n",
       "3  CN104785960-A    CN10150108    31 Mar 2015;   ...   \n",
       "4  WO2017043583-A1    WOJP076473    08 Sep 2016; ...   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  transport apparatus vacuum transport system ar...  \n",
       "1  automatic injection mold machine feeding syste...  \n",
       "2  fully-automatic single-row planting robot rota...  \n",
       "3  grid structure type robot weld device robot pr...  \n",
       "4  robot apparatus industrial application operati...  "
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.nltk预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import pandas as pd\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 去掉纯数字\n",
    "def is_hex(word):\n",
    "    word= word.lower()\n",
    "    for i in range(len(word)):\n",
    "        if not( word[i].startswith('0x') or word[i].isdigit() or word[i] in ['a','b','c','d','e','f']):\n",
    "            return False\n",
    "    return True\n",
    "\n",
    "# def is_not_num(x):\n",
    "#     return not( len(x) > 0 and (x.isnumeric() or x[0].isdigit() or is_hex(x)) )\n",
    "\n",
    "def is_not_num(x):\n",
    "    return not( len(x) > 0 and (x.isnumeric() or x[0].isdigit() ) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>PN 专利号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>AD 申请详细信息</th>\n",
       "      <th>预处理文本</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>patent5534</td>\n",
       "      <td>US2015071737-A1;  US10424498-B2</td>\n",
       "      <td>Transport apparatus for vacuum transport syste...</td>\n",
       "      <td>US2015071737-A1    US480803    09 Sep 2014;   ...</td>\n",
       "      <td>transport apparatus vacuum transport system ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>patent31677</td>\n",
       "      <td>CN107498781-A;  CN207290719-U;  CN107498781-B</td>\n",
       "      <td>Automatic injection molding machine feeding   ...</td>\n",
       "      <td>CN107498781-A    CN10664563    04 Aug 2017;   ...</td>\n",
       "      <td>automatic injection mold machine feeding syste...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>patent83440</td>\n",
       "      <td>CN108029284-A;  CN207869642-U</td>\n",
       "      <td>Fully-automatic single-row planting robot, has...</td>\n",
       "      <td>CN108029284-A    CN10027388    11 Jan 2018;   ...</td>\n",
       "      <td>fully-automatic single-row planting robot rota...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>patent10376</td>\n",
       "      <td>CN104785960-A;  CN104785960-B</td>\n",
       "      <td>Grid structure type robot welding device has r...</td>\n",
       "      <td>CN104785960-A    CN10150108    31 Mar 2015;   ...</td>\n",
       "      <td>grid structure type robot weld device robot pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>patent12420</td>\n",
       "      <td>WO2017043583-A1;  TW201718203-A;  JP2017539221...</td>\n",
       "      <td>Robot apparatus for use in industrial applicat...</td>\n",
       "      <td>WO2017043583-A1    WOJP076473    08 Sep 2016; ...</td>\n",
       "      <td>robot apparatus industrial application operati...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            编号                                             PN 专利号  \\\n",
       "0   patent5534                    US2015071737-A1;  US10424498-B2   \n",
       "1  patent31677      CN107498781-A;  CN207290719-U;  CN107498781-B   \n",
       "2  patent83440                      CN108029284-A;  CN207869642-U   \n",
       "3  patent10376                      CN104785960-A;  CN104785960-B   \n",
       "4  patent12420  WO2017043583-A1;  TW201718203-A;  JP2017539221...   \n",
       "\n",
       "                                             TI 文献标题  \\\n",
       "0  Transport apparatus for vacuum transport syste...   \n",
       "1  Automatic injection molding machine feeding   ...   \n",
       "2  Fully-automatic single-row planting robot, has...   \n",
       "3  Grid structure type robot welding device has r...   \n",
       "4  Robot apparatus for use in industrial applicat...   \n",
       "\n",
       "                                           AD 申请详细信息  \\\n",
       "0  US2015071737-A1    US480803    09 Sep 2014;   ...   \n",
       "1  CN107498781-A    CN10664563    04 Aug 2017;   ...   \n",
       "2  CN108029284-A    CN10027388    11 Jan 2018;   ...   \n",
       "3  CN104785960-A    CN10150108    31 Mar 2015;   ...   \n",
       "4  WO2017043583-A1    WOJP076473    08 Sep 2016; ...   \n",
       "\n",
       "                                               预处理文本  \n",
       "0  transport apparatus vacuum transport system ar...  \n",
       "1  automatic injection mold machine feeding syste...  \n",
       "2  fully-automatic single-row planting robot rota...  \n",
       "3  grid structure type robot weld device robot pr...  \n",
       "4  robot apparatus industrial application operati...  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = data.copy()\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 69273/69273 [00:47<00:00, 1461.64it/s]\n"
     ]
    }
   ],
   "source": [
    "# 分词\n",
    "from nltk import word_tokenize\n",
    "token_ls = []\n",
    "text_ls = []\n",
    "for sentence in tqdm(df['预处理文本']):\n",
    "    token_words = word_tokenize(sentence)\n",
    "    text = ' '.join(token_words)\n",
    "    token_ls.append(token_words)   # list of list 分词结果\n",
    "    text_ls.append(text)           # list 分词列表转化为文本"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers\\punkt.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 再小写\n",
    "little_ls = []\n",
    "for words_list in token_ls:\n",
    "    little_words= [x.lower() for x in words_list]\n",
    "    little_ls.append(little_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████| 69273/69273 [15:17<00:00, 75.51it/s]\n"
     ]
    }
   ],
   "source": [
    "# 词性标注\n",
    "from nltk import pos_tag\n",
    "token_pos_ls = []\n",
    "for token in tqdm(little_ls):\n",
    "    pos_words = pos_tag(token)     #词性标注\n",
    "    token_pos_ls.append(pos_words)\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Unzipping taggers\\averaged_perceptron_tagger.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('averaged_perceptron_tagger')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 69273/69273 [00:51<00:00, 1337.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "51.81198191642761\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# 词形归一化\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "\n",
    "huanyuan_ls = []\n",
    "wordnet_lematizer = WordNetLemmatizer()\n",
    "t1 = time.time()\n",
    "for token_words in tqdm(token_pos_ls):\n",
    "    words_lematizer = []\n",
    "    for word, tag in token_words:\n",
    "        if tag.startswith('NN'):\n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='n')  # n代表名词\n",
    "        elif tag.startswith('VB'): \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='v')   # v代表动词\n",
    "        elif tag.startswith('JJ'): \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='a')   # a代表形容词\n",
    "        elif tag.startswith('R'): \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word, pos='r')   # r代表代词\n",
    "        else: \n",
    "            word_lematizer =  wordnet_lematizer.lemmatize(word)\n",
    "        words_lematizer.append(word_lematizer)\n",
    "    huanyuan_ls.append(words_lematizer)  # 得到还原列表\n",
    "#     break\n",
    "t2 = time.time()\n",
    "print(t2-t1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('wordnet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package omw-1.4 to\n",
      "[nltk_data]     C:\\Users\\zb\\AppData\\Roaming\\nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# import nltk\n",
    "# nltk.download('omw-1.4')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 去停用词\n",
    "with open(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\3技术研究\\英文停用词.txt','r') as f:\n",
    "    stop_ls = f.readlines()\n",
    "stop_ls = [i.strip() for i in stop_ls]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "clean_ls =[]\n",
    "for words_ls in huanyuan_ls:\n",
    "    cleaned_words = [word for word in words_ls if word not in stop_ls]\n",
    "    clean_ls.append(cleaned_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████| 69273/69273 [00:03<00:00, 18808.89it/s]\n"
     ]
    }
   ],
   "source": [
    "# 去掉纯数字\n",
    "last_text = []\n",
    "for text in tqdm(clean_ls):\n",
    "    res = filter(is_not_num, text)\n",
    "    res = ' '.join(filter(is_not_num, text))\n",
    "    last_text.append(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 删除长度小于等于2的单词\n",
    "new_text_ls = []\n",
    "for textls in last_text:\n",
    "    new_text_ls.append(\" \".join([word for word in textls.split() if len(word) > 2]))\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['预处理文本2'] = new_text_ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>编号</th>\n",
       "      <th>PN 专利号</th>\n",
       "      <th>TI 文献标题</th>\n",
       "      <th>AD 申请详细信息</th>\n",
       "      <th>预处理文本</th>\n",
       "      <th>预处理文本2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>patent5534</td>\n",
       "      <td>US2015071737-A1;  US10424498-B2</td>\n",
       "      <td>Transport apparatus for vacuum transport syste...</td>\n",
       "      <td>US2015071737-A1    US480803    09 Sep 2014;   ...</td>\n",
       "      <td>transport apparatus vacuum transport system ar...</td>\n",
       "      <td>transport apparatus vacuum transport system ar...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>patent31677</td>\n",
       "      <td>CN107498781-A;  CN207290719-U;  CN107498781-B</td>\n",
       "      <td>Automatic injection molding machine feeding   ...</td>\n",
       "      <td>CN107498781-A    CN10664563    04 Aug 2017;   ...</td>\n",
       "      <td>automatic injection mold machine feeding syste...</td>\n",
       "      <td>automatic injection mold machine feed system c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>patent83440</td>\n",
       "      <td>CN108029284-A;  CN207869642-U</td>\n",
       "      <td>Fully-automatic single-row planting robot, has...</td>\n",
       "      <td>CN108029284-A    CN10027388    11 Jan 2018;   ...</td>\n",
       "      <td>fully-automatic single-row planting robot rota...</td>\n",
       "      <td>fully-automatic single-row planting robot rota...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>patent10376</td>\n",
       "      <td>CN104785960-A;  CN104785960-B</td>\n",
       "      <td>Grid structure type robot welding device has r...</td>\n",
       "      <td>CN104785960-A    CN10150108    31 Mar 2015;   ...</td>\n",
       "      <td>grid structure type robot weld device robot pr...</td>\n",
       "      <td>grid structure type robot weld device robot pr...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>patent12420</td>\n",
       "      <td>WO2017043583-A1;  TW201718203-A;  JP2017539221...</td>\n",
       "      <td>Robot apparatus for use in industrial applicat...</td>\n",
       "      <td>WO2017043583-A1    WOJP076473    08 Sep 2016; ...</td>\n",
       "      <td>robot apparatus industrial application operati...</td>\n",
       "      <td>robot apparatus industrial application operati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69268</th>\n",
       "      <td>patent88505</td>\n",
       "      <td>CN110053976-A;  CN210150293-U</td>\n",
       "      <td>Automatic glass receiving and collecting machi...</td>\n",
       "      <td>CN110053976-A    CN10484784    04 Jun 2019;   ...</td>\n",
       "      <td>automatic glass receiving collect machine conv...</td>\n",
       "      <td>automatic glass receive collect machine convey...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69269</th>\n",
       "      <td>patent88509</td>\n",
       "      <td>CN108190483-A;  CN108190483-B</td>\n",
       "      <td>Medicine dispensing robot mounting device asse...</td>\n",
       "      <td>CN108190483-A    CN11465973    28 Dec 2017;   ...</td>\n",
       "      <td>medicine dispense robot mount device assembly ...</td>\n",
       "      <td>medicine dispense robot mount device assembly ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69270</th>\n",
       "      <td>patent88516</td>\n",
       "      <td>WO2022194768-A1;  EP4060007-A1</td>\n",
       "      <td>Composite material used in flexible device,   ...</td>\n",
       "      <td>WO2022194768-A1    WOEP056520    14 Mar 2022; ...</td>\n",
       "      <td>composite material flexible device preferably ...</td>\n",
       "      <td>composite material flexible device preferably ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69271</th>\n",
       "      <td>patent88520</td>\n",
       "      <td>KR2406840-B1</td>\n",
       "      <td>Exoskeleton device for lower extremity rehabil...</td>\n",
       "      <td>KR2406840-B1    KR005897    14 Jan 2022</td>\n",
       "      <td>exoskeleton device low extremity rehabilitatio...</td>\n",
       "      <td>exoskeleton device low extremity rehabilitatio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69272</th>\n",
       "      <td>patent88522</td>\n",
       "      <td>CN114343852-A</td>\n",
       "      <td>Operating device for use in medical device of ...</td>\n",
       "      <td>CN114343852-A    CN10103288    27 Jan 2022</td>\n",
       "      <td>operating device medical device surgical robot...</td>\n",
       "      <td>operating device medical device surgical robot...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>69273 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                编号                                             PN 专利号  \\\n",
       "0       patent5534                    US2015071737-A1;  US10424498-B2   \n",
       "1      patent31677      CN107498781-A;  CN207290719-U;  CN107498781-B   \n",
       "2      patent83440                      CN108029284-A;  CN207869642-U   \n",
       "3      patent10376                      CN104785960-A;  CN104785960-B   \n",
       "4      patent12420  WO2017043583-A1;  TW201718203-A;  JP2017539221...   \n",
       "...            ...                                                ...   \n",
       "69268  patent88505                      CN110053976-A;  CN210150293-U   \n",
       "69269  patent88509                      CN108190483-A;  CN108190483-B   \n",
       "69270  patent88516                     WO2022194768-A1;  EP4060007-A1   \n",
       "69271  patent88520                                       KR2406840-B1   \n",
       "69272  patent88522                                      CN114343852-A   \n",
       "\n",
       "                                                 TI 文献标题  \\\n",
       "0      Transport apparatus for vacuum transport syste...   \n",
       "1      Automatic injection molding machine feeding   ...   \n",
       "2      Fully-automatic single-row planting robot, has...   \n",
       "3      Grid structure type robot welding device has r...   \n",
       "4      Robot apparatus for use in industrial applicat...   \n",
       "...                                                  ...   \n",
       "69268  Automatic glass receiving and collecting machi...   \n",
       "69269  Medicine dispensing robot mounting device asse...   \n",
       "69270  Composite material used in flexible device,   ...   \n",
       "69271  Exoskeleton device for lower extremity rehabil...   \n",
       "69272  Operating device for use in medical device of ...   \n",
       "\n",
       "                                               AD 申请详细信息  \\\n",
       "0      US2015071737-A1    US480803    09 Sep 2014;   ...   \n",
       "1      CN107498781-A    CN10664563    04 Aug 2017;   ...   \n",
       "2      CN108029284-A    CN10027388    11 Jan 2018;   ...   \n",
       "3      CN104785960-A    CN10150108    31 Mar 2015;   ...   \n",
       "4      WO2017043583-A1    WOJP076473    08 Sep 2016; ...   \n",
       "...                                                  ...   \n",
       "69268  CN110053976-A    CN10484784    04 Jun 2019;   ...   \n",
       "69269  CN108190483-A    CN11465973    28 Dec 2017;   ...   \n",
       "69270  WO2022194768-A1    WOEP056520    14 Mar 2022; ...   \n",
       "69271            KR2406840-B1    KR005897    14 Jan 2022   \n",
       "69272         CN114343852-A    CN10103288    27 Jan 2022   \n",
       "\n",
       "                                                   预处理文本  \\\n",
       "0      transport apparatus vacuum transport system ar...   \n",
       "1      automatic injection mold machine feeding syste...   \n",
       "2      fully-automatic single-row planting robot rota...   \n",
       "3      grid structure type robot weld device robot pr...   \n",
       "4      robot apparatus industrial application operati...   \n",
       "...                                                  ...   \n",
       "69268  automatic glass receiving collect machine conv...   \n",
       "69269  medicine dispense robot mount device assembly ...   \n",
       "69270  composite material flexible device preferably ...   \n",
       "69271  exoskeleton device low extremity rehabilitatio...   \n",
       "69272  operating device medical device surgical robot...   \n",
       "\n",
       "                                                  预处理文本2  \n",
       "0      transport apparatus vacuum transport system ar...  \n",
       "1      automatic injection mold machine feed system c...  \n",
       "2      fully-automatic single-row planting robot rota...  \n",
       "3      grid structure type robot weld device robot pr...  \n",
       "4      robot apparatus industrial application operati...  \n",
       "...                                                  ...  \n",
       "69268  automatic glass receive collect machine convey...  \n",
       "69269  medicine dispense robot mount device assembly ...  \n",
       "69270  composite material flexible device preferably ...  \n",
       "69271  exoskeleton device low extremity rehabilitatio...  \n",
       "69272  operating device medical device surgical robot...  \n",
       "\n",
       "[69273 rows x 6 columns]"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(r'D:\\jupyter\\DK\\DK数据\\2数据处理\\3聚类\\3技术研究\\2新语料.xlsx',index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
